import requests
from bs4 import BeautifulSoup
import urllib.request
import os
import time,threading

def LoadUrl(url):
	headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
	'Accept-Encoding':'gzip, deflate, sdch',
	'Accept-Language':'zh-CN,zh;q=0.8',
	'Connection':'keep-alive',
	'Cookie':'UM_distinctid=15c9f8884a74c-0b916f07a-434f032d-1fa400-15c9f8884abef; CNZZDATA1261954307=1764696472-1497340231-https%253A%252F%252Fwww.doutula.com%252F%7C1497340231; XSRF-TOKEN=eyJpdiI6IjRiUHZjd2U1YTFVdG5NWmVkdjkrS0E9PSIsInZhbHVlIjoiNW91RFJCVmRCWWFnQ00xOHc3ZEtaRmVVbkpLV0M3b3c1MG1JMnhBdEsyMmFKUE5rYmxwMUJRODNDVkxoaWJUc2h6aFFNMTlDdU5TeHlpdmJIZVwvd1wvUT09IiwibWFjIjoiYjc4ZTA4N2JiYTM0ZjRiNDllNmE5MDc2NDhjM2M2MTdjMGMzMWU0NTVlOTdmY2QxOWQ0ZGViZDJmMmYzODBjYyJ9; laravel_session=eyJpdiI6ImRyNUpLcUw1OTk5bDExZzMxenpoeVE9PSIsInZhbHVlIjoicmJ4NVgzS0ZqczdOd0ZsVlc0MW5ORTZNZnh4c1NEcXE0dTNDcjFHRE9oYzY3SVQ1WnliOE1JbTZBMXorRjA2RjVMWEdFc1wvTEQzRnJwUjJzVzZJelh3PT0iLCJtYWMiOiI1MzUzZjRiNzFkODZiZjU3OGNlMThkOGQ5N2E5Yzg0MDcxYzM2ODBiNjhhYjEyZjFkODhlZDM2OTk3ZDc3ZWNjIn0%3D; CNZZDATA1256911977=1427953587-1497320860-https%253A%252F%252Fwww.baidu.com%252F%7C1497958946; _ga=GA1.2.65797461.1497325143; _gid=GA1.2.1737994871.1497961002',
	'Host':'www.doutula.com',
	'Upgrade-Insecure-Requests':'1',
	'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.130 Safari/537.36'
	}
	r = requests.get(url,headers)
	return r.text

def GetImage(html):
	soup=BeautifulSoup(html,'html.parser')
	pagecontent= soup.find('div',attrs={'class':'page-content'})
	imageList= pagecontent.find_all('a', attrs={}, recursive=True, text=None, limit=None, kwargs='')
	images=[]
	for item in imageList:
		src= item.find_all('img')
		for temp in src:
			if temp.has_attr('data-original'):
				imageurl=temp.get('data-original')
				images.append('https:'+imageurl)
	return images


def loadImage(imageList,index):
	file='D:\py\Buiness\pic\\newface'
	for x in imageList:
		try:
			fs=x.split('/')
			fn=fs[len(fs)-1]
			newfile=''
			newfile=file+'\\'+str(index)
			if  os.path.exists(newfile)!=True:
				print("no")
				os.makedirs(newfile)
			newfile=newfile+'\\'+fn


			# urllib.request.urlretrieve(url,newfile)
			print(newfile)
			req=urllib.request.urlopen(x)
			buf = req.read()
			with open(newfile,'wb') as f:
				f.write(buf)
		except Exception as e:
			raise e
		finally:
			pass

def turn(url,index):
	for x in range(84,index):
		try:
			htmlUrl = url+str(x)
			print(htmlUrl)
			content=LoadUrl(htmlUrl)
			imageList=GetImage(content)
			loadImage(imageList,x)
		except Exception as e:
			print(e)
		else:
			pass
		finally:
			print(x)

url="https://www.doutula.com/photo/list/?page="
turn(url,500)

# content=LoadUrl(url)
# imageList=GetImage(content)
# imageList=['https://ws3.sinaimg.cn/bmiddle/9150e4e5ly1fgqtevylbaj202v01naa0.jpg']


	


